# 从apt源爬取元数据，转换成json文件存储

import urllib.request
import os
import gzip
import json
import argparse



# 选项列表
ubuntu_release_names = ["bionic", "devel", "focal", "hirsute", "impish", "jammy", "kinetic", "trusty", "xenial"]
archs = ["binary-amd64/", "binary-i386/"]
# 更新方式：默认版与不同安全程度的更新版
ubuntu_update_types = ["/", "-backports/", "-security/", "-updates/", "-proposed/"]
# 四种软件包仓库
ubuntu_package_types = ["main/", "multiverse/", "restricted/", "universe/"]
ubuntu_dists_url = "http://us.archive.ubuntu.com/ubuntu/dists/"


# 将给定位置的元数据内容存入指定json文件
def transform2Json(meta_file_path, release_name, update_type, package_type):
    json_file_path = meta_file_path + ".json"
    f = open(meta_file_path, "r", encoding="utf-8")
    package_list = f.read().split("\n\n")
    json_dict = {}  # 最终存储json文件的字典
    for package in package_list:
        items = {}  # 存储当前包的{条目名：条目信息}
        item_list = package.split("\n")
        for item in item_list:  # 依次遍历每个条目
            new_item = item.split(":", 1)
            if len(new_item) < 2:  # 这是防止非条目干扰，如空行等
                continue
            items[new_item[0].strip()] = new_item[1].strip()
        if items == {}:
            continue
        try:
            package_name = items['Package']  # 取出当前包名
        except KeyError:  # 如果没有包名？打印出来看看啥情况
            print("++++++++++Error: Missing Package Name++++++++++")
            print(items)
        items["Ubuntu-Release-Name"] = release_name
        items["Package-Update-Type"] = update_type if update_type != "" else "basic"
        items["Package-Type"] = package_type
        json_dict[package_name] = items
    with open(json_file_path, "w", encoding='utf-8') as f1:  ## 设置'utf-8'编码
        f1.write(json.dumps(json_dict, ensure_ascii=False, indent=2, separators=(',', ': '))) # (先转换成字符串再存储)
    f.close()


# 把指定的ubuntu发行版(及架构)存入指定文件夹
def meta_miner(release_name, arch, store_path):
    for update_type in ubuntu_update_types:
        for package_type in ubuntu_package_types:
            file_name = release_name + update_type + package_type + arch
            file_url = ubuntu_dists_url + file_name + "/" + "Packages.gz"
            print(file_url)
            meta_file_path = store_path + file_name.replace("/", "_")  # 存储在本地时的名字
            gz_file_path = store_path + file_name.replace("/", "_") + ".gz"
            urllib.request.urlretrieve(file_url, gz_file_path)  # 获取网站中的元数据，gz压缩包形式
            g_file = gzip.GzipFile(gz_file_path)  # 读压缩包
            # 把压缩包中的内容写入store_path位置的file_name文件（压缩包中只有一个元数据文件）
            open(meta_file_path, "wb+").write(g_file.read())
            g_file.close()
            transform2Json(meta_file_path, release_name,
                           update_type.replace("/", "").replace("-", ""),
                           package_type.replace("/", ""))


def mkdir(Folder):
    if not os.path.exists(Folder):  # 是否存在这个文件夹
        os.makedirs(Folder)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()  # 创建一个解析对象
    # 向该对象中添加要关注的命令行参数和选项
    parser.add_argument('--store_path', default='./xenial')
    parser.add_argument('--release_name', default='xenial')
    parser.add_argument('--arch', default='binary-amd64')
    args = parser.parse_args()
    mkdir(args.store_path)
    meta_miner(args.release_name, args.arch, args.store_path + "/")

